/*---------------------------------------------------------------------------
  NAME:
    ascii_read

  CALLING SEQUENCE:
    IDL> struct = ascii_read(file/lun, structdef, numrows, 
                             rows=, columns=, /csv, status=, verbose=, /help)
  
  PURPOSE: 

    Read ASCII data from file into a structure.  The file can be white space or
    comma separated value (CSV). The structdef input provides the definition
    describing each row of the file.  Particular rows or columns may be
    extracted by number. For very large files, this is a big advantage over the
    IDL readu procedure which can only read contiguous chunks.  The return
    variable is a structure containing the requested data.

    Unlike binary_read, the columns may be variable length and the user can
    input the string columns in structdef with any size because the memory will
    be generated on the fly.  E.g. structdef = {a:0L, b:'', c:0LL}. String
    columns are currently limited to 255 bytes.

    The user can send the file name, in which case the data must start at the
    beginning of the file, or the file unit opened from IDL.

    In general, due to the complex inputs and the fact that most files will
    have a header describing the data, this program will be used as a utility
    program and an IDL wrapper will parse the header and format the structure
    definition.

    This program is written in C and is linked to IDL via the DLM mechanism.


  INPUTS: 
     file/lun: Filename or file unit. For string file names, the user must 
               expand all ~ or other environment variables.  If the file
	       unit is entered, the file must be opened with the appropriate 
	       keywords:
                 openr, lun, file, /get_lun, /stdio, bufsize=0

     structdef: A structure that describes the layout of the data in each row.
                Variable length fields are not supported.
     numrows: Number of rows in the file.

  OPTIONAL INPUTS:
     rows=: An array or scalar of unique rows to read
     columns=: An array or scalar of unique columns numbers to extract.
     /csv: The file is formatted as comma separated value.  The fields cannot 
           contain commas in this case.
     verbose=: 0 for standard quiet. 1 for basic info. > 1 for debug mode.
     /help: print this message.

  OPTIONAL OUTPUTS:
    status=: The status of the read. 0 for success, 1 for read failure, 
             2 for input errors such as bad file unit.

  REVISION HISTORY:
    created 20-April-2006: Erin Sheldon, NYU



  Copyright (C) 2005  Erin Sheldon, NYU.  erin dot sheldon at gmail dot com

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA



  ---------------------------------------------------------------------------*/

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "export.h"
#include "fileio_util.h"
#include "ascii_read.h"

KW_RESULT kw;
/* int8 verbose; */

IDL_VPTR
ascii_read(int argc, IDL_VPTR *argv, char *argk)
{

  /* Input structure, file, nrows, rows=, columns= */
  IDL_VPTR instructVptr, fileVptr, nrowsVptr;
  char *file;
  IDL_VPTR rowsVptr;
  IDL_MEMINT nrows, row, skiprows, 
    row2read, nrows2read, irow2read; 
  IDL_MEMINT_VECTOR *rows;

  FILE *FILEPTR;

  /* offsets will depend on whether or not we are requesting columns */
  IDL_MEMINT *tagOffsets;

  IDL_VPTR columnsVptr;
  int8 *get_column;
  int16 *columns;
  IDL_MEMINT ncolumns;

  /* Tag information */

  idl_tag_info *input_tag_info, *tag_info;
  int32 input_tag, tag;

  IDL_VPTR resultVptr, tmpVptr;
  UCHAR *dataPtr, *tptr, *rowptr;

  IDL_STRING *tStringPtr;

  IDL_STRUCT_TAG_DEF *tagdefs;
  int nparams;
  UCHAR tagtype;

  /* Buffer for reading skipped columns and buffering string data*/
  char buffer[255];
  int ret;

  /* verbosity level */
  int verbose=0, debug=0;

  /* Loop variables */
  IDL_MEMINT i;

  /* messages */
  char mess[100];
  int read_error;

  /* Will hold formats for reading */
  char **scan_formats;
  int csv=0;

  ar_set_status(READ_FAILURE);



  /* Process the keywords */
  (void) IDL_KWProcessByOffset(argc, argv, argk, kw_pars, 
			       (IDL_VPTR *) 0, 1, &kw);


  /* Print detailed documentation if requested */
  if (kw.help_there) {
    if (kw.help) {
      ar_syntax(1);
      return(IDL_GettmpInt(-1));
    }
  }

  if (kw.csv_there) 
    csv = kw.csv;

  /* Check # of parameters */
  nparams = ar_nparams(argc);
  if (nparams < 3) {
    ar_syntax(0);
    return(IDL_GettmpInt(-1));
  }



  /* Get inputs */
  fileVptr = argv[0];
  instructVptr = argv[1];
  nrowsVptr = argv[2];



  /* Get the file pointer. */
  if ( !(FILEPTR = get_fileptr(fileVptr,"r")) )
    {
      ar_set_status(INPUT_ERROR);      
      return(IDL_GettmpInt(-1));  
    }

  /* Get number of rows to read */
  nrows = IDL_MEMINTScalar(nrowsVptr);

  if (kw.verbose_there)
    {
      verbose = kw.verbose;
      if (verbose > 1) debug=1;
    }



  /*
   * Keyword Data 
   */

  /* Did user request certain rows? */
  if ( kw.rows_there )
    {
      rows = getrows(kw.rows, nrows);
      if (rows == NULL)
	{
	  error_message("Keyword rows contains no valid rows",NULL);
	  ar_set_status(INPUT_ERROR);      
	  return(IDL_GettmpInt(-1));  
	}

      nrows2read = rows->n;
      if (verbose) {
	sprintf(mess,"Extracting %d/%d rows",nrows2read,nrows);
	message(mess);
      }
    }
  else 
    {
      nrows2read = nrows;
      if (verbose) {
	sprintf(mess,"Reading %d rows",nrows2read);
	message(mess);
      }
    }

  /* Get tag info from the input structure */
  input_tag_info = get_idl_tag_info(instructVptr, verbose);


  /* Did the user request certain columns? This will determine the
     output */
  /* WARNING currently columns must be numbers and these must be sorted 
     and unique! */

  if (debug) message("Getting columns");//;fflush(stdout);
  dataPtr = get_output(instructVptr,
		       kw.columns_there, kw.columns, 
		       input_tag_info,
		       nrows2read,
		       &resultVptr,
		       &tagOffsets,
		       &get_column, verbose);
  if (dataPtr == NULL)
    {
      ar_set_status(INPUT_ERROR);
      free_tag_info(input_tag_info);
      if (fileVptr->type == IDL_TYP_STRING)
	fclose(FILEPTR);


      return(IDL_GettmpInt(-1));  

    }
  if (debug) message("Done columns");//;fflush(stdout);

  /* Get the scan formats */
  scan_formats = ar_scan_formats(csv);

 
  /* The calls above are identical to those in binary_read.  Maybe consolidate. */

  /* Read the data. */
  read_error = 0;
  row = 0;

  for (irow2read=0; irow2read<nrows2read; irow2read++)
    {


      /* Are we selecting rows? If not, then we just read */
      if (kw.rows_there)
	{
	  row2read = rows->d[irow2read];

	  /* Now see if we need to more ahead in the file */
	  if (row < row2read)
	    {
	      /* If not the first row, we must also grab the \n from
		 the last line */
	      if (irow2read == 0)
		skiprows = row2read - row;
	      else
		skiprows = row2read - row + 1;
	      if (!ar_skiplines(FILEPTR, skiprows))
		{
		  message("Requested rows beyond EOF");
		  read_error=1;
		  break;
		}
	      row = row2read;
	    }
	}

      /* Loop over all available tags */
      tag = 0;
      for (input_tag=0;input_tag<input_tag_info->ntags; input_tag++)
	{
	  
	  /* Are we saving this info in output struct? */
	  tagtype = input_tag_info->tagDesc[input_tag]->type;	      
	  if (get_column[input_tag])
	    {
	      
	      tptr = 
		( resultVptr->value.s.arr->data +
		  irow2read*( resultVptr->value.arr->elt_len ) + 
		  tagOffsets[tag]);
	      
	      if (tagtype != IDL_TYP_STRING)
		{
		  /* loop over elements */
		  for (i=0; i<input_tag_info->tagNelts[input_tag]; i++)
		    {

		      ret = ar_scanval(FILEPTR, scan_formats[tagtype], tptr);
		      if (ret != 1)
			{
			  read_error=1;
			  break;
			}
		      tptr += input_tag_info->tagNbytes[input_tag];
		    }
		}
	      else 
		{
		  /* Need to loop over and store each separately in a buffer
		     before copying to struct.  This will work for scalars and
		     arrays of strings. */
		  
		  tStringPtr = (IDL_STRING *) tptr;
		  for (i=0; i<input_tag_info->tagNelts[input_tag]; i++)
		    {
		      
		      ret = ar_scanval(FILEPTR, scan_formats[tagtype], 
				       buffer);
		      if (ret != 1)
			{
			  read_error=1;
			  break;
			}
		      
		      IDL_StrStore(tStringPtr, buffer);
		      tStringPtr++;
		    }
		  
		}
	      /* Separately increment copied tags */
	      tag++;
		  
	    } /* Copying column into output variable */
	  else
	    {
	      /* Skipping this column */
	      for (i=0;i<input_tag_info->tagNelts[input_tag]; i++)
		{
		  ret = ar_scanval(FILEPTR, scan_formats[tagtype], buffer);
		  if (ret != 1)
		    {
		      read_error=1;
		      break;
		    }
		  
		}
	      
	    } /* Not keeping this column */
	  
	  if (read_error)
	    break;
	  
	} /* Loop over columns */


      if (read_error)
	{
	  error_message_where(row,input_tag);
	  break;
	}
      
      row++;

    } /* Loop over rows*/


  ar_free_scan_formats(scan_formats);
  free(get_column);
  free(tagOffsets);
  free_tag_info(input_tag_info);

  if (kw.rows_there)
    IDL_MEMINT_VECTOR_Free(rows);
  
  if (fileVptr->type == IDL_TYP_STRING)
    fclose(FILEPTR);
  
  
  IDL_KW_FREE;
  
  /* Was the read successful? */
  if (read_error)
    {
      /* I've decided it is better to return what we have read to help the user
	 diagnose the problem */
      ar_set_status(READ_FAILURE);
      return(resultVptr);
      //IDL_Deltmp(resultVptr);
      //return(IDL_GettmpInt(-1));  
    }
  else
    {
      ar_set_status(READ_SUCCESS);
      return(resultVptr);
    }

}


/*
  ar_scanval: read from the stream and print error messages.
 */

int ar_scanval(FILE *fptr, char *format, char *buffer)
{
  int ret;

  ret = fscanf(fptr, format, buffer);
  if (ret != 1)
    {
      if (feof(fptr))
	  error_message("EOF reached unexpectedly",NULL);
      else
	{
	  char mess[50];
	  sprintf(mess,"Read error: %d",ret);
	  error_message(mess,NULL);
	}
    }

  return(ret);
}

/* For error checking mainly */
void ar_printval(int code, char *membuf)
{

  switch (code)
    {
    case 1:
      printf("%d", *( (UCHAR *) membuf )); // idl_byte
      break;
    case 2:
      printf("%d", *( (short *) membuf )); // idl_int
      break;
    case 3:
      printf("%d", *( (IDL_LONG *) membuf )); // idl_long
      break;
    case 4: 
      printf("%f", *( (float *) membuf)); // idl_float
      break;
    case 5:
      printf("%lf", *( (double *) membuf)); // idl_double
      break;
    case 7:
      printf("%s", membuf); // idl_string
      break;
    case 12:
      printf("%d", *( (unsigned short *) membuf)); // idl_uint
      break;
    case 13:
      printf("%d", *( (IDL_ULONG *) membuf)); // idl_ulong
      break;
    case 14:
      printf("%Ld", *( (IDL_LONG64 *) membuf)); // idl_long64
      break;
    case 15:
      printf("%Lu", *( (IDL_ULONG64 *) membuf)); // idl_ulong64
      break;
    default:
      {
	char mess[50];
	sprintf(mess,"Cannot print type: %d",code);
	message(mess);
      }
      break;
    }
  printf("\n");
  fflush(stdout);
}

int ar_skiplines(FILE *fptr, int nskip)
{
  int i;
  char c;

  for (i=0;i<nskip;i++)
    {
      while ((c = fgetc(fptr)) != '\n' && c != EOF)
	;
      
      if (feof(fptr))
	{
	  error_message("EOF reached unexpectedly",NULL);
	  return(0);
	}
      
    }

  return(1);
}


/* The index of the format numbers corresponds directly to the type number,
   so we can simply index this with that type. Note, not all are supported,
   such as complex types, pointers, objref, struct*/
char **ar_scan_formats(int csv)
{

  char **scan_formats;

  int nformat = IDL_NUM_TYPES;
  int i;
  int size_format;

  scan_formats = calloc(nformat, sizeof(char *));
  size_format = 6;
  for (i=0;i<nformat;i++)
    scan_formats[i] = calloc(size_format, sizeof(char));

  if (csv) 
    {

      /* Need spaces before comma to allow for them in file 
	 This works because scanf treats blanks as zero or more
	 white space */
      strcpy(scan_formats[0],"NONE"); // Undef
      strcpy(scan_formats[1],"%d ,"); // idl_byte
      strcpy(scan_formats[2],"%d ,"); // idl_int
      strcpy(scan_formats[3],"%d ,"); // idl_long
      strcpy(scan_formats[4],"%f ,"); // idl_float
      strcpy(scan_formats[5],"%lf ,"); // idl_double
      strcpy(scan_formats[6],"NONE"); // idl_complex
      strcpy(scan_formats[7],"%[^,],"); // idl_string
      strcpy(scan_formats[8],"NONE"); // idl_struct
      strcpy(scan_formats[9],"NONE"); // idl_double_complex
      strcpy(scan_formats[10],"NONE"); // idl_ptr
      strcpy(scan_formats[11],"NONE"); // idl_objref
      strcpy(scan_formats[12],"%u ,"); // idl_uint
      strcpy(scan_formats[13],"%u ,"); // idl_ulong
      strcpy(scan_formats[14],"%Ld ,"); // idl_long64
      strcpy(scan_formats[15],"%Lu ,"); // idl_ulong64

    }
  else
    {

      strcpy(scan_formats[0],"NONE"); // Undef
      strcpy(scan_formats[1],"%d"); // idl_byte
      strcpy(scan_formats[2],"%d"); // idl_int
      strcpy(scan_formats[3],"%d"); // idl_long
      strcpy(scan_formats[4],"%f"); // idl_float
      strcpy(scan_formats[5],"%lf"); // idl_double
      strcpy(scan_formats[6],"NONE"); // idl_comples
      strcpy(scan_formats[7],"%s"); // idl_string
      strcpy(scan_formats[8],"NONE"); // idl_struct
      strcpy(scan_formats[9],"NONE"); // idl_double_complex
      strcpy(scan_formats[10],"NONE"); // idl_ptr
      strcpy(scan_formats[11],"NONE"); // idl_objref
      strcpy(scan_formats[12],"%u"); // idl_uint
      strcpy(scan_formats[13],"%u"); // idl_ulong
      strcpy(scan_formats[14],"%Ld"); // idl_long64
      strcpy(scan_formats[15],"%Lu"); // idl_ulong64

    }
    
  return(scan_formats);
}

void
ar_free_scan_formats(char **scan_formats)
{
  int i;
  for (i=0;i<IDL_NUM_TYPES;i++)
    free(scan_formats[i]);

  free(scan_formats);

}

int ar_nparams(int argc)
{

  int nKeywords;

  nKeywords =
    kw.columns_there + 
    kw.csv_there + 
    kw.help_there +
    kw.rows_there + 
    kw.status_there + 
    kw.verbose_there;

  return 
    argc - nKeywords;
}


/* Put in main because uses kw */
void
ar_set_status(int statusVal)
{

  if (kw.status_there) {
    /* This frees any existing memory and sets type to INT with value zero */
    IDL_StoreScalarZero(kw.status, IDL_TYP_INT);
    kw.status->value.i = statusVal;

  }

}


void ar_syntax(int help)
{



  if (help) 
    {

      char docstring[] = "/*---------------------------------------------------------------------------\n  NAME:\n    ascii_read\n\n  CALLING SEQUENCE:\n    IDL> struct = ascii_read(file/lun, structdef, numrows, \n                             rows=, columns=, /csv, status=, verbose=, /help)\n  \n  PURPOSE: \n\n    Read ASCII data from file into a structure.  The file can be white space or\n    comma separated value (CSV). The user inputs the structure definition that\n    describes each row of the file.  The user can extract columns and rows by\n    number. For very large files, this is a big advantage over the IDL read\n    procedure which can only read contiguous chunks.  The return variable is a\n    structure containing the requested data.\n\n    Unlike binary_read, the columns may be variable length and the user can\n    input the string columns in structdef with any size because the memory will\n    be generated on the fly.  E.g. structdef = {a:0L, b:'', c:0LL}. String\n    columns are currently limited to 255 bytes.\n\n    The user can send the file name, in which case the data must start at the\n    beginning of the file, or the file unit opened from IDL.\n\n    In general, due to the complex inputs and the fact that most files will\n    have a header describing the data, this program will be used as a utility\n    program and an IDL wrapper will parse the header and format the structure\n    definition.\n\n    This program is written in C and is linked to IDL via the DLM mechanism.\n\n\n  INPUTS: \n     file/lun: Filename or file unit. For string file names, the user must \n               expand all ~ or other environment variables.  If the file\n           unit is entered, the file must be opened with the appropriate \n        keywords:\n                 openr, lun, file, /get_lun, /stdio, bufsize=0\n\n     structdef: A structure that describes the layout of the data in each row.\n                Variable length fields are not supported.\n     numrows: Number of rows in the file.\n\n  OPTIONAL INPUTS:\n     rows=: An array or scalar of unique rows to read\n     columns=: An array or scalar of unique columns numbers to extract.\n     /csv: The file is formatted as comma separated value.  The fields cannot \n           contain commas in this case.\n     verbose=: 0 for standard quiet. 1 for basic info. > 1 for debug mode.\n     /help: print this message.\n\n  OPTIONAL OUTPUTS:\n    status=: The status of the read. 0 for success, 1 for read failure, \n             2 for input errors such as bad file unit.\n\n  REVISION HISTORY:\n    created 20-April-2006: Erin Sheldon, NYU\n\n\n\n  Copyright (C) 2005  Erin Sheldon, NYU.  erin dot sheldon at gmail dot com\n\n    This program is free software; you can redistribute it and/or modify\n    it under the terms of the GNU General Public License as published by\n    the Free Software Foundation; either version 2 of the License, or\n    (at your option) any later version.\n\n    This program is distributed in the hope that it will be useful,\n    but WITHOUT ANY WARRANTY; without even the implied warranty of\n    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n    GNU General Public License for more details.\n\n    You should have received a copy of the GNU General Public License\n    along with this program; if not, write to the Free Software\n    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA\n\n\n\n  ---------------------------------------------------------------------------*/";

      printf("%s\n", docstring);
    }
  else
    {
      printf("\n");
      printf("   struct = ascii_read(file/lun, structdef, numrows, \n");
      printf("                       rows=, columns=, /csv, status=, verbose=, /help)\n");
      printf("\n");
      printf("   Send /help for full documentation.\n");
    }
  fflush(stdout);

}



#define ARRLEN(arr) (sizeof(arr)/sizeof(arr[0]))

int IDL_Load(void)
{

  /* This must be static. It is a struct. */
  /* The name in strings is the name by which it will be called from IDL and
     MUST BE CAPITALIZED 
     5th parameter will say if it accepts keywords and some other flags 
     For more info see page 325 of external dev. guide */
  static IDL_SYSFUN_DEF2 func_addr[] = {
    { (IDL_SYSRTN_GENERIC) ascii_read, "ASCII_READ", 
      0, IDL_MAXPARAMS, 
      IDL_SYSFUN_DEF_F_KEYWORDS, 0},
  };

  /* False means it is not a function */
  return IDL_SysRtnAdd(func_addr, IDL_TRUE, ARRLEN(func_addr));

}
